package com.xiaotu.spider.tvtropes.pageprocessor;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.*;
import com.xiaotu.spider.AUrlPageProcessor;
import org.apache.commons.lang.StringUtils;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.util.List;
import java.util.Map;

/**
 * 桥段数据爬取
 *
 * @auther GaoHJ
 * @create 2017/11/1
 */
@Component("TVTropesPageProcessor")
public class TVTropesPageProcessor extends AUrlPageProcessor
{
    private static final String STORY_FLAG = "/Main/";//桥段标识

    private static final String STATIC_PATH = "static";//无script的html文件路径

    private static final String DYMANIC_PATH = "dynamic";//完整html文件路径

    private String filePath;

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("filePath") == null)
            throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
        filePath = map.get("filePath") + StringUtils.EMPTY;
    }

    @Override
    public void process(Page page)
    {
        if (Constants.PageDataType.TV_RECORD.equals(this.getPageDataType(page)))
            this.storyProcess(page);
        else
            this.indexProcess(page);
    }

    private void indexProcess(Page page)
    {
        this.addStoryLink(page, "//div[@class='page-content " +
                "index-page']//a[@class='twikilink']/@href", true);
    }


    private void addStoryLink(Page page, String xpath, boolean passed)
    {
        List<String> linkList = page.getHtml().xpath(xpath).all();
        if (linkList == null || linkList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        MD5 md5 = new MD5();
        linkList.stream().filter(link -> (passed) || (!FileUtil.fileExits(this.filePath + File
                .separator + DYMANIC_PATH + File.separator + md5.getMD5(link) + ".html") && link
                .contains(STORY_FLAG))).forEach(link -> PageUtils.addTargetRequest(page, link,
                page.getRequest().getExtras()));
    }

    private void storyProcess(Page page)
    {
        String storyTitle = StringUtils.trim(page.getHtml().xpath("//h1[@class='page-title']/text" +
                "()").get());
        if (StringUtils.isEmpty(storyTitle))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);

        String oldFileName = new MD5().getMD5(page.getUrl().toString()) + ".html";
        String fileName = new MD5().getMD5(storyTitle) + ".html";
        String dFileName = this.filePath + File.separator + DYMANIC_PATH + File.separator +
                fileName;
        if (FileUtil.fileExits(dFileName) || FileUtil.fileExits(oldFileName))
            return;
        String sFileName = this.filePath + File.separator + STATIC_PATH + File.separator + fileName;
        FileUtil.writeFile(dFileName, page.getRawText());
        FileUtil.writeFile(sFileName, HtmlUtil.delHTMLTag(page.getRawText(), HtmlUtil
                .regEx_script));
        FileUtil.writeFile(this.filePath + File.separator + "story.csv", storyTitle +
                SepratorUtil.SEP_PIPE + fileName + SepratorUtil.SEP_PIPE + page.getUrl()
                .toString() + SepratorUtil.SEP_PIPE + System.currentTimeMillis() +
                SepratorUtil.LINE_SEPRATOR);
        this.addStoryLink(page, "//a[@class='twikilink']/@href", false);
    }


    @Override
    protected String[] getStartRequestUrls()
    {
        return new String[]{"http://tvtropes.org/pmwiki/index_report.php"};
    }

    @Override
    protected String getSpiderType()
    {
        return "Story";
    }

    @Override
    public String getPageDataType(Page page)
    {
        if (page.getUrl().toString().contains(STORY_FLAG))
            return Constants.PageDataType.TV_RECORD;
        return Constants.PageDataType.SEARCH_LIST;
    }
}
